{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/sklearn/externals/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n",
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "import category_encoders as ce\n",
    "from contextlib import contextmanager\n",
    "import lightgbm as lgb\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from scipy.cluster.vq import kmeans2, whiten\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.decomposition import truncated_svd\n",
    "import category_encoders as ce\n",
    "from catboost import CatBoostClassifier, CatBoostRegressor\n",
    "from sklearn import preprocessing\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "num_rows = None\n",
    "EPS = 1e-100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "descretize = lambda x, n: list(map(str, list(pd.qcut(x, n, duplicates='drop'))))\n",
    "\n",
    "def binary_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.BinaryEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df[new_columns]\n",
    "\n",
    "def helmert_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.HelmertEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df[new_columns]\n",
    "\n",
    "def target_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.TargetEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    return df[categorical_columns]\n",
    "\n",
    "def poly_encoder(df, n_train):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    enc = ce.PolynomialEncoder(impute_missing=True, cols=categorical_columns).fit(df[0:n_train], df[0:n_train]['TARGET'])\n",
    "    df = enc.transform(df)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df[new_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"Level_1_stack_new/test_ann_a_1.csv\"\n",
    "validation_file_path = 'Level_1_stack_new/validation_ann_a_1.csv'\n",
    "num_folds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/SureshFeaturesAug16_2.csv', header=0, index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AMT_GOODS_PRICE_TO_CREDIT</th>\n",
       "      <th>AMT_CREDIT_INSTALLMENTS</th>\n",
       "      <th>AMT_CREDIT_INTEREST_MAYBE</th>\n",
       "      <th>AMT_CREDIT_INTEREST_MAYBE_RAT</th>\n",
       "      <th>AMT_INCOME_TOTAL_RAT_CREDIT</th>\n",
       "      <th>NAME_CONTRACT_TYPE_Consumer_loans</th>\n",
       "      <th>NAME_CONTRACT_TYPE_Cash_loans</th>\n",
       "      <th>NAME_CONTRACT_TYPE_Revolving_loans</th>\n",
       "      <th>NAME_CONTRACT_TYPE_XNA</th>\n",
       "      <th>PRE_RECORD_COUNT</th>\n",
       "      <th>...</th>\n",
       "      <th>STCK_PAY_720L_.</th>\n",
       "      <th>STCK_PAY_180L_.</th>\n",
       "      <th>STCK_PAY_480_.</th>\n",
       "      <th>STCK_CC_6_.</th>\n",
       "      <th>STCK_BERBAL_6_.</th>\n",
       "      <th>EXT_MEAN</th>\n",
       "      <th>TERM_BEFORE_END</th>\n",
       "      <th>DAYS_DECISION_MEAN</th>\n",
       "      <th>AMT_CREDIT_LIMIT_ACTUAL_6</th>\n",
       "      <th>AMT_CREDIT_LIMIT_ACTUAL_12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.863262</td>\n",
       "      <td>16.461104</td>\n",
       "      <td>11389.5</td>\n",
       "      <td>0.028012</td>\n",
       "      <td>0.144444</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.080220</td>\n",
       "      <td>0.076729</td>\n",
       "      <td>0.080736</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.160535</td>\n",
       "      <td>0.161787</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-606.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.873211</td>\n",
       "      <td>36.234085</td>\n",
       "      <td>8356.5</td>\n",
       "      <td>0.006460</td>\n",
       "      <td>0.348611</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053463</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-3915.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-815.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.949845</td>\n",
       "      <td>10.532818</td>\n",
       "      <td>15817.5</td>\n",
       "      <td>0.050586</td>\n",
       "      <td>0.183333</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.081061</td>\n",
       "      <td>0.069552</td>\n",
       "      <td>0.073751</td>\n",
       "      <td>0.03353</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-2452.0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>270000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>23.461618</td>\n",
       "      <td>10093.5</td>\n",
       "      <td>0.019675</td>\n",
       "      <td>0.351852</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.074988</td>\n",
       "      <td>0.090350</td>\n",
       "      <td>0.076197</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-7337.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 268 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   AMT_GOODS_PRICE_TO_CREDIT  AMT_CREDIT_INSTALLMENTS  \\\n",
       "0                   0.863262                16.461104   \n",
       "1                   0.873211                36.234085   \n",
       "2                   1.000000                20.000000   \n",
       "3                   0.949845                10.532818   \n",
       "4                   1.000000                23.461618   \n",
       "\n",
       "   AMT_CREDIT_INTEREST_MAYBE  AMT_CREDIT_INTEREST_MAYBE_RAT  \\\n",
       "0                    11389.5                       0.028012   \n",
       "1                     8356.5                       0.006460   \n",
       "2                        0.0                       0.000000   \n",
       "3                    15817.5                       0.050586   \n",
       "4                    10093.5                       0.019675   \n",
       "\n",
       "   AMT_INCOME_TOTAL_RAT_CREDIT  NAME_CONTRACT_TYPE_Consumer_loans  \\\n",
       "0                     0.144444                                0.0   \n",
       "1                     0.348611                                1.0   \n",
       "2                     0.166667                                0.0   \n",
       "3                     0.183333                                5.0   \n",
       "4                     0.351852                                4.0   \n",
       "\n",
       "   NAME_CONTRACT_TYPE_Cash_loans  NAME_CONTRACT_TYPE_Revolving_loans  \\\n",
       "0                            1.0                                 0.0   \n",
       "1                            2.0                                 0.0   \n",
       "2                            1.0                                 0.0   \n",
       "3                            2.0                                 2.0   \n",
       "4                            2.0                                 0.0   \n",
       "\n",
       "   NAME_CONTRACT_TYPE_XNA  PRE_RECORD_COUNT             ...              \\\n",
       "0                     0.0               1.0             ...               \n",
       "1                     0.0               3.0             ...               \n",
       "2                     0.0               1.0             ...               \n",
       "3                     0.0               9.0             ...               \n",
       "4                     0.0               6.0             ...               \n",
       "\n",
       "   STCK_PAY_720L_.  STCK_PAY_180L_.  STCK_PAY_480_.  STCK_CC_6_.  \\\n",
       "0         0.080220         0.076729        0.080736          NaN   \n",
       "1         0.053463              NaN             NaN          NaN   \n",
       "2              NaN              NaN             NaN          NaN   \n",
       "3         0.081061         0.069552        0.073751      0.03353   \n",
       "4         0.074988         0.090350        0.076197          NaN   \n",
       "\n",
       "   STCK_BERBAL_6_.  EXT_MEAN  TERM_BEFORE_END  DAYS_DECISION_MEAN  \\\n",
       "0         0.160535  0.161787              1.0              -606.0   \n",
       "1              NaN       NaN              3.0             -3915.0   \n",
       "2              NaN       NaN              1.0              -815.0   \n",
       "3              NaN       NaN              2.0             -2452.0   \n",
       "4              NaN       NaN              4.0             -7337.0   \n",
       "\n",
       "   AMT_CREDIT_LIMIT_ACTUAL_6  AMT_CREDIT_LIMIT_ACTUAL_12  \n",
       "0                        NaN                         NaN  \n",
       "1                        NaN                         NaN  \n",
       "2                        NaN                         NaN  \n",
       "3                   270000.0                    270000.0  \n",
       "4                        NaN                         NaN  \n",
       "\n",
       "[5 rows x 268 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "goran_features = pd.read_csv('../goran-data/goranm_feats_v3.csv', header=0, index_col=None)\n",
    "del goran_features['SK_ID_CURR']\n",
    "del goran_features['IS_TRAIN']\n",
    "goran_features.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniques = [f for f in goran_features.columns if f not in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, goran_features[uniques]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_features2 = pd.read_csv('../data/selected_features2.csv', header=0, index_col=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniques = [f for f in new_features2.columns if f not in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, new_features2[uniques]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 816)"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.externals import joblib\n",
    "\n",
    "shubin_df = joblib.load('../data/stacked_featureSet')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>solution1_pred_lgb</th>\n",
       "      <th>solution2_pred_lgb</th>\n",
       "      <th>bureau_v2_solution</th>\n",
       "      <th>bureau_solution2</th>\n",
       "      <th>application_solution2</th>\n",
       "      <th>pos_v1_solution2</th>\n",
       "      <th>prev_application_solution2</th>\n",
       "      <th>EXT1</th>\n",
       "      <th>EXT2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>0.218051</td>\n",
       "      <td>0.264317</td>\n",
       "      <td>0.105383</td>\n",
       "      <td>0.092550</td>\n",
       "      <td>0.438352</td>\n",
       "      <td>0.079057</td>\n",
       "      <td>0.086284</td>\n",
       "      <td>0.234845</td>\n",
       "      <td>0.230231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0.012417</td>\n",
       "      <td>0.018419</td>\n",
       "      <td>0.025442</td>\n",
       "      <td>0.028613</td>\n",
       "      <td>0.047676</td>\n",
       "      <td>0.061434</td>\n",
       "      <td>0.017387</td>\n",
       "      <td>0.013653</td>\n",
       "      <td>0.019212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0.036228</td>\n",
       "      <td>0.039559</td>\n",
       "      <td>0.041481</td>\n",
       "      <td>0.055773</td>\n",
       "      <td>0.043101</td>\n",
       "      <td>0.077291</td>\n",
       "      <td>0.119400</td>\n",
       "      <td>0.042936</td>\n",
       "      <td>0.036041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0.040368</td>\n",
       "      <td>0.047467</td>\n",
       "      <td>0.101213</td>\n",
       "      <td>0.101191</td>\n",
       "      <td>0.045045</td>\n",
       "      <td>0.092093</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>0.036698</td>\n",
       "      <td>0.040683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0.051251</td>\n",
       "      <td>0.044134</td>\n",
       "      <td>0.048242</td>\n",
       "      <td>0.056467</td>\n",
       "      <td>0.095363</td>\n",
       "      <td>0.039969</td>\n",
       "      <td>0.078971</td>\n",
       "      <td>0.046074</td>\n",
       "      <td>0.056351</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  solution1_pred_lgb  solution2_pred_lgb  bureau_v2_solution  \\\n",
       "0      100002            0.218051            0.264317            0.105383   \n",
       "1      100003            0.012417            0.018419            0.025442   \n",
       "2      100004            0.036228            0.039559            0.041481   \n",
       "3      100006            0.040368            0.047467            0.101213   \n",
       "4      100007            0.051251            0.044134            0.048242   \n",
       "\n",
       "   bureau_solution2  application_solution2  pos_v1_solution2  \\\n",
       "0          0.092550               0.438352          0.079057   \n",
       "1          0.028613               0.047676          0.061434   \n",
       "2          0.055773               0.043101          0.077291   \n",
       "3          0.101191               0.045045          0.092093   \n",
       "4          0.056467               0.095363          0.039969   \n",
       "\n",
       "   prev_application_solution2      EXT1      EXT2  \n",
       "0                    0.086284  0.234845  0.230231  \n",
       "1                    0.017387  0.013653  0.019212  \n",
       "2                    0.119400  0.042936  0.036041  \n",
       "3                    0.083527  0.036698  0.040683  \n",
       "4                    0.078971  0.046074  0.056351  "
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shubin_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniques = [f for f in shubin_df.columns if f not in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, shubin_df[uniques]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 825)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../data/application_train.csv.zip', nrows=num_rows)\n",
    "n_train = train.shape[0]\n",
    "test = pd.read_csv('../data/application_test.csv.zip', nrows=num_rows)\n",
    "    \n",
    "new_df = pd.concat([train, test], axis=0)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns = [col for col in train.columns if train[col].dtype == 'object']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = preprocessing.LabelEncoder()\n",
    "\n",
    "for f in categorical_columns:\n",
    "    if new_df[f].dtype == 'object':\n",
    "        new_df[f] = encoder.fit_transform(new_df[f].apply(str).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def do_countuniq( df, group_cols, counted, agg_name, agg_type='uint32', show_max=False, show_agg=True ):\n",
    "    if show_agg:\n",
    "        print( \"Counting unqiue \", counted, \" by \", group_cols , '...' )\n",
    "    gp = df[group_cols+[counted]].groupby(group_cols)[counted].nunique().reset_index().rename(columns={counted:agg_name})\n",
    "    df = df.merge(gp, on=group_cols, how='left')\n",
    "    del gp\n",
    "    if show_max:\n",
    "        print( agg_name + \" max value = \", df[agg_name].max() )\n",
    "    df[agg_name] = df[agg_name].astype(agg_type)\n",
    "    gc.collect()\n",
    "    return df \n",
    "\n",
    "def do_mean(df, group_cols, counted, agg_name, agg_type='float32', show_max=False, show_agg=True ):\n",
    "    if show_agg:\n",
    "        print( \"Calculating mean of \", counted, \" by \", group_cols , '...' )\n",
    "    gp = df[group_cols+[counted]].groupby(group_cols)[counted].mean().reset_index().rename(columns={counted:agg_name})\n",
    "    df = df.merge(gp, on=group_cols, how='left')\n",
    "    del gp\n",
    "    if show_max:\n",
    "        print( agg_name + \" max value = \", df[agg_name].max() )\n",
    "    df[agg_name] = df[agg_name].astype(agg_type)\n",
    "    gc.collect()\n",
    "    return df \n",
    "\n",
    "def do_count(df, group_cols, agg_name, agg_type='uint32', show_max=False, show_agg=True ):\n",
    "    if show_agg:\n",
    "        print( \"Aggregating by \", group_cols , '...' )\n",
    "    gp = df[group_cols][group_cols].groupby(group_cols).size().rename(agg_name).to_frame().reset_index()\n",
    "    df = df.merge(gp, on=group_cols, how='left')\n",
    "    del gp\n",
    "    if show_max:\n",
    "        print( agg_name + \" max value = \", df[agg_name].max() )\n",
    "    df[agg_name] = df[agg_name].astype(agg_type)\n",
    "    gc.collect()\n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counting unqiue  CODE_GENDER  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['NAME_TYPE_SUITE'] ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME_TYPE_SUITE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-ORGANIZATION_TYPE_cunique max value =  57\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['OCCUPATION_TYPE'] ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OCCUPATION_TYPE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-NAME_INCOME_TYPE_cunique max value =  7\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['HOUSETYPE_MODE'] ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HOUSETYPE_MODE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-WALLSMATERIAL_MODE_cunique max value =  8\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  EMERGENCYSTATE_MODE  by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE-EMERGENCYSTATE_MODE_cunique max value =  3\n",
      "Counting unqiue  NAME_CONTRACT_TYPE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-NAME_CONTRACT_TYPE_cunique max value =  2\n",
      "Counting unqiue  CODE_GENDER  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-CODE_GENDER_cunique max value =  3\n",
      "Counting unqiue  FLAG_OWN_CAR  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-FLAG_OWN_CAR_cunique max value =  2\n",
      "Counting unqiue  FLAG_OWN_REALTY  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-FLAG_OWN_REALTY_cunique max value =  2\n",
      "Counting unqiue  NAME_TYPE_SUITE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-NAME_TYPE_SUITE_cunique max value =  8\n",
      "Counting unqiue  NAME_INCOME_TYPE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-NAME_INCOME_TYPE_cunique max value =  8\n",
      "Counting unqiue  NAME_EDUCATION_TYPE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-NAME_EDUCATION_TYPE_cunique max value =  5\n",
      "Counting unqiue  NAME_FAMILY_STATUS  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-NAME_FAMILY_STATUS_cunique max value =  6\n",
      "Counting unqiue  NAME_HOUSING_TYPE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-NAME_HOUSING_TYPE_cunique max value =  6\n",
      "Counting unqiue  OCCUPATION_TYPE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-OCCUPATION_TYPE_cunique max value =  19\n",
      "Counting unqiue  WEEKDAY_APPR_PROCESS_START  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-WEEKDAY_APPR_PROCESS_START_cunique max value =  7\n",
      "Counting unqiue  ORGANIZATION_TYPE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-ORGANIZATION_TYPE_cunique max value =  58\n",
      "Counting unqiue  FONDKAPREMONT_MODE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-FONDKAPREMONT_MODE_cunique max value =  5\n",
      "Counting unqiue  HOUSETYPE_MODE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-HOUSETYPE_MODE_cunique max value =  4\n",
      "Counting unqiue  WALLSMATERIAL_MODE  by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE-WALLSMATERIAL_MODE_cunique max value =  8\n"
     ]
    }
   ],
   "source": [
    "counts_columns = []\n",
    "for f_0 in categorical_columns:\n",
    "    for f_1 in [x for x in categorical_columns if x != f_0] :\n",
    "        new_df = do_countuniq(new_df, [f_0], f_1,\n",
    "                      f_0 + '-' + f_1 + '_cunique', 'uint16', show_max=True); gc.collect()\n",
    "        counts_columns.append(f_0 + '-' + f_1 + '_cunique')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Aggregating by  ['NAME_CONTRACT_TYPE'] ...\n",
      "NAME_CONTRACT_TYPE_count max value =  326537\n",
      "Aggregating by  ['CODE_GENDER'] ...\n",
      "CODE_GENDER_count max value =  235126\n",
      "Aggregating by  ['FLAG_OWN_CAR'] ...\n",
      "FLAG_OWN_CAR_count max value =  235235\n",
      "Aggregating by  ['FLAG_OWN_REALTY'] ...\n",
      "FLAG_OWN_REALTY_count max value =  246970\n",
      "Aggregating by  ['NAME_TYPE_SUITE'] ...\n",
      "NAME_TYPE_SUITE_count max value =  288253\n",
      "Aggregating by  ['NAME_INCOME_TYPE'] ...\n",
      "NAME_INCOME_TYPE_count max value =  183307\n",
      "Aggregating by  ['NAME_EDUCATION_TYPE'] ...\n",
      "NAME_EDUCATION_TYPE_count max value =  252379\n",
      "Aggregating by  ['NAME_FAMILY_STATUS'] ...\n",
      "NAME_FAMILY_STATUS_count max value =  228715\n",
      "Aggregating by  ['NAME_HOUSING_TYPE'] ...\n",
      "NAME_HOUSING_TYPE_count max value =  316513\n",
      "Aggregating by  ['OCCUPATION_TYPE'] ...\n",
      "OCCUPATION_TYPE_count max value =  111996\n",
      "Aggregating by  ['WEEKDAY_APPR_PROCESS_START'] ...\n",
      "WEEKDAY_APPR_PROCESS_START_count max value =  63652\n",
      "Aggregating by  ['ORGANIZATION_TYPE'] ...\n",
      "ORGANIZATION_TYPE_count max value =  78832\n",
      "Aggregating by  ['FONDKAPREMONT_MODE'] ...\n",
      "FONDKAPREMONT_MODE_count max value =  243092\n",
      "Aggregating by  ['HOUSETYPE_MODE'] ...\n",
      "HOUSETYPE_MODE_count max value =  177916\n",
      "Aggregating by  ['WALLSMATERIAL_MODE'] ...\n",
      "WALLSMATERIAL_MODE_count max value =  180234\n",
      "Aggregating by  ['EMERGENCYSTATE_MODE'] ...\n",
      "EMERGENCYSTATE_MODE_count max value =  185607\n"
     ]
    }
   ],
   "source": [
    "count_columns = []\n",
    "for f_0 in categorical_columns:\n",
    "        new_df = do_count(new_df, [f_0],\n",
    "                      f_0  + '_count', 'uint16', show_max=True); gc.collect()\n",
    "        count_columns.append(f_0  + '_count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = [f for f in new_df.columns if f not in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df, new_df[feats]], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 1202)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_drop = [\n",
    "\n",
    "  \"FLAG_DOCUMENT_2\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_7\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_10\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_12\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_13\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_14\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_15\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_16\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_17\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_18\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_19\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_20\",\n",
    "\n",
    "  \"FLAG_DOCUMENT_21\",\n",
    "\n",
    "  \"PREV_NAME_CONTRACT_TYPE_Consumer_loans\",\n",
    "\n",
    "  \"PREV_NAME_CONTRACT_TYPE_XNA\",\n",
    "\n",
    "  \"PB_CNT_NAME_CONTRACT_STATUS_Amortized_debt\",\n",
    "\n",
    "  \"MAX_DATA_ALL\",\n",
    "\n",
    "  \"MIN_DATA_ALL\",\n",
    "\n",
    "  \"MAX_MIN_DURATION\",\n",
    "\n",
    "  \"MAX_AMT_CREDIT_MAX_OVERDUE\",\n",
    "\n",
    "  \"CC_AMT_DRAWINGS_ATM_CURRENT_MIN\",\n",
    "\n",
    "  \"CC_AMT_DRAWINGS_OTHER_CURRENT_MAX\",\n",
    "\n",
    "  \"CC_AMT_DRAWINGS_OTHER_CURRENT_MIN\",\n",
    "\n",
    "  \"CC_CNT_DRAWINGS_ATM_CURRENT_MIN\",\n",
    "\n",
    "  \"CC_CNT_DRAWINGS_OTHER_CURRENT_MAX\",\n",
    "\n",
    "  \"CC_CNT_DRAWINGS_OTHER_CURRENT_MIN\",\n",
    "\n",
    "  \"CC_SK_DPD_DEF_MIN\",\n",
    "\n",
    "  \"CC_SK_DPD_MIN\",\n",
    "\n",
    "  \"BERB_STATUS_CREDIT_TYPE_Loan_for_working_capital_replenishment\",\n",
    "\n",
    " \"BERB_STATUS_CREDIT_TYPE_Real_estate_loan\",\n",
    "\n",
    "  \"BERB_STATUS_CREDIT_TYPE_Loan_for_the_purchase_of_equipment\",\n",
    "\n",
    "  \"BERB_COMBO_CT_CA_COMBO_CT_CA_Loan_for_working_capital_replenishmentClosed\",\n",
    "\n",
    "  \"BERB_COMBO_CT_CA_COMBO_CT_CA_Car_loanSold\",\n",
    "\n",
    "  \"BERB_COMBO_CT_CA_COMBO_CT_CA_Another_type_of_loanActive\",\n",
    "\n",
    "  \"BERB_COMBO_CT_CA_COMBO_CT_CA_Loan_for_working_capital_replenishmentSold\",\n",
    "\n",
    "  \"BERB_COMBO_CT_CA_COMBO_CT_CA_MicroloanSold\",\n",
    "\n",
    "  \"BERB_COMBO_CT_CA_COMBO_CT_CA_Another_type_of_loanSold\",\n",
    "\n",
    "  \"FLAG_EMAIL\",\n",
    "\n",
    "  \"APARTMENTS_AVG\",\n",
    "\n",
    "  \"AMT_REQ_CREDIT_BUREAU_MON\",\n",
    "\n",
    "  \"AMT_REQ_CREDIT_BUREAU_QRT\",\n",
    "\n",
    "  \"AMT_REQ_CREDIT_BUREAU_YEAR\",\n",
    "\n",
    "  \"STCK_BERBAL_6_\",\n",
    "\n",
    "  \"STCK_CC_6_x\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = [f for f in cols_to_drop if f in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(labels=feats, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 1160)"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "@contextmanager\n",
    "def timer(name):\n",
    "    t0 = time.time()\n",
    "    yield\n",
    "    print('[{' + name + '}] done in {' + str(round(time.time() - t0, 3)) + '} s')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from keras.callbacks import Callback\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "import keras as ks\n",
    "from keras import backend as K\n",
    "import gc\n",
    "from contextlib import contextmanager\n",
    "import tensorflow as tf\n",
    "\n",
    "class RocAucEvaluation(Callback):\n",
    "    def __init__(self, validation_data=(), interval=1):\n",
    "        super(Callback, self).__init__()\n",
    "\n",
    "        self.interval = interval\n",
    "        self.X_val, self.y_val = validation_data\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if epoch % self.interval == 0:\n",
    "            y_pred = self.model.predict(self.X_val, verbose=0)\n",
    "            score = roc_auc_score(self.y_val, y_pred)\n",
    "            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting ANN. Train shape: (307511, 1160), test shape: (48744, 1160)\n",
      "(307511, 1158)\n",
      "(246008, 1161) (61503, 1161) (48744, 1161)\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/20\n",
      "246008/246008 [==============================] - 65s 262us/step - loss: 0.2689 - binary_crossentropy: 0.2689 - val_loss: 0.2524 - val_binary_crossentropy: 0.2524\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.755923\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.25244, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246008/246008 [==============================] - 63s 255us/step - loss: 0.2492 - binary_crossentropy: 0.2492 - val_loss: 0.2495 - val_binary_crossentropy: 0.2495\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.762791\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.25244 to 0.24953, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246008/246008 [==============================] - 63s 255us/step - loss: 0.2470 - binary_crossentropy: 0.2470 - val_loss: 0.2494 - val_binary_crossentropy: 0.2494\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.766710\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24953 to 0.24939, saving model to best_model.hdf5\n",
      "Epoch 4/20\n",
      "246008/246008 [==============================] - 63s 255us/step - loss: 0.2458 - binary_crossentropy: 0.2458 - val_loss: 0.2531 - val_binary_crossentropy: 0.2531\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.765929\n",
      "\n",
      "Epoch 00004: val_loss did not improve from 0.24939\n",
      "[{pass 1}] done in {290.265} s\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/20\n",
      "246008/246008 [==============================] - 58s 237us/step - loss: 0.2442 - binary_crossentropy: 0.2442 - val_loss: 0.2483 - val_binary_crossentropy: 0.2483\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.769538\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24939 to 0.24826, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246008/246008 [==============================] - 58s 236us/step - loss: 0.2432 - binary_crossentropy: 0.2432 - val_loss: 0.2469 - val_binary_crossentropy: 0.2469\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.770767\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24826 to 0.24687, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246008/246008 [==============================] - 58s 236us/step - loss: 0.2428 - binary_crossentropy: 0.2428 - val_loss: 0.2480 - val_binary_crossentropy: 0.2480\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.771018\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24687\n",
      "[{pass 2}] done in {201.752} s\n",
      "Train on 246008 samples, validate on 61503 samples\n",
      "Epoch 1/20\n",
      "246008/246008 [==============================] - 58s 235us/step - loss: 0.2425 - binary_crossentropy: 0.2425 - val_loss: 0.2466 - val_binary_crossentropy: 0.2466\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.772213\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24687 to 0.24659, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246008/246008 [==============================] - 58s 235us/step - loss: 0.2418 - binary_crossentropy: 0.2418 - val_loss: 0.2462 - val_binary_crossentropy: 0.2462\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.773107\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24659 to 0.24616, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246008/246008 [==============================] - 58s 235us/step - loss: 0.2419 - binary_crossentropy: 0.2419 - val_loss: 0.2464 - val_binary_crossentropy: 0.2464\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.772614\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24616\n",
      "[{pass 3}] done in {200.987} s\n",
      "0.7726139312958156\n",
      "[{fit_predict}] done in {719.261} s\n",
      "(246009, 1161) (61502, 1161) (48744, 1161)\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/20\n",
      "246009/246009 [==============================] - 65s 264us/step - loss: 0.2691 - binary_crossentropy: 0.2691 - val_loss: 0.2477 - val_binary_crossentropy: 0.2477\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.762464\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.24769, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246009/246009 [==============================] - 63s 255us/step - loss: 0.2501 - binary_crossentropy: 0.2501 - val_loss: 0.2468 - val_binary_crossentropy: 0.2468\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.764994\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24769 to 0.24680, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246009/246009 [==============================] - 63s 255us/step - loss: 0.2474 - binary_crossentropy: 0.2474 - val_loss: 0.2464 - val_binary_crossentropy: 0.2464\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.766464\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.24680 to 0.24642, saving model to best_model.hdf5\n",
      "Epoch 4/20\n",
      "246009/246009 [==============================] - 63s 256us/step - loss: 0.2473 - binary_crossentropy: 0.2473 - val_loss: 0.2444 - val_binary_crossentropy: 0.2444\n",
      "\n",
      " ROC-AUC - epoch: 4 - score: 0.768735\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.24642 to 0.24438, saving model to best_model.hdf5\n",
      "Epoch 5/20\n",
      "246009/246009 [==============================] - 63s 255us/step - loss: 0.2458 - binary_crossentropy: 0.2458 - val_loss: 0.2457 - val_binary_crossentropy: 0.2457\n",
      "\n",
      " ROC-AUC - epoch: 5 - score: 0.771425\n",
      "\n",
      "Epoch 00005: val_loss did not improve from 0.24438\n",
      "[{pass 1}] done in {365.655} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/20\n",
      "246009/246009 [==============================] - 60s 242us/step - loss: 0.2444 - binary_crossentropy: 0.2444 - val_loss: 0.2428 - val_binary_crossentropy: 0.2428\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.774097\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24438 to 0.24280, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246009/246009 [==============================] - 58s 235us/step - loss: 0.2432 - binary_crossentropy: 0.2432 - val_loss: 0.2425 - val_binary_crossentropy: 0.2425\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.774311\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24280 to 0.24251, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246009/246009 [==============================] - 58s 236us/step - loss: 0.2430 - binary_crossentropy: 0.2430 - val_loss: 0.2431 - val_binary_crossentropy: 0.2431\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.773873\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24251\n",
      "[{pass 2}] done in {204.278} s\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/20\n",
      "246009/246009 [==============================] - 59s 240us/step - loss: 0.2426 - binary_crossentropy: 0.2426 - val_loss: 0.2420 - val_binary_crossentropy: 0.2420\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.775632\n",
      "\n",
      "Epoch 00001: val_loss improved from 0.24251 to 0.24197, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246009/246009 [==============================] - 58s 234us/step - loss: 0.2424 - binary_crossentropy: 0.2424 - val_loss: 0.2418 - val_binary_crossentropy: 0.2418\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.776499\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24197 to 0.24176, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "246009/246009 [==============================] - 58s 234us/step - loss: 0.2421 - binary_crossentropy: 0.2421 - val_loss: 0.2420 - val_binary_crossentropy: 0.2420\n",
      "\n",
      " ROC-AUC - epoch: 3 - score: 0.776833\n",
      "\n",
      "Epoch 00003: val_loss did not improve from 0.24176\n",
      "[{pass 3}] done in {203.137} s\n",
      "0.7768330403729315\n",
      "[{fit_predict}] done in {799.506} s\n",
      "(246009, 1161) (61502, 1161) (48744, 1161)\n",
      "Train on 246009 samples, validate on 61502 samples\n",
      "Epoch 1/20\n",
      "246009/246009 [==============================] - 64s 262us/step - loss: 0.2704 - binary_crossentropy: 0.2704 - val_loss: 0.2456 - val_binary_crossentropy: 0.2456\n",
      "\n",
      " ROC-AUC - epoch: 1 - score: 0.751163\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.24558, saving model to best_model.hdf5\n",
      "Epoch 2/20\n",
      "246009/246009 [==============================] - 63s 256us/step - loss: 0.2506 - binary_crossentropy: 0.2506 - val_loss: 0.2436 - val_binary_crossentropy: 0.2436\n",
      "\n",
      " ROC-AUC - epoch: 2 - score: 0.756877\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.24558 to 0.24362, saving model to best_model.hdf5\n",
      "Epoch 3/20\n",
      "243712/246009 [============================>.] - ETA: 0s - loss: 0.2484 - binary_crossentropy: 0.2484"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-153-8c9921fe2829>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     68\u001b[0m                         \u001b[0;32mwith\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'pass '\u001b[0m \u001b[0;34m+\u001b[0m  \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m                             model.fit(x=x_train, y=train['TARGET'].iloc[train_idx].values, batch_size=batch_size+(batch_size*(2*i)), epochs=20, \n\u001b[0;32m---> 70\u001b[0;31m                                 validation_data=(x_valid, train['TARGET'].iloc[valid_idx].values), callbacks=[ra_val, check_point, early_stop], shuffle=True)\n\u001b[0m\u001b[1;32m     71\u001b[0m                     \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_valid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     72\u001b[0m                     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mroc_auc_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'TARGET'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mvalid_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_score\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1703\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1704\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1705\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1706\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1707\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1234\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1236\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1237\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2480\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2481\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2482\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2483\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1140\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1141\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1142\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1321\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1325\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1310\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1311\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1312\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1313\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1314\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Workstation/PyEnv/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1418\u001b[0m         return tf_session.TF_Run(\n\u001b[1;32m   1419\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1420\u001b[0;31m             status, run_metadata)\n\u001b[0m\u001b[1;32m   1421\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1422\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "encoding = 'ohe'\n",
    "\n",
    "train_df = df.iloc[0:n_train]\n",
    "test_df = df.iloc[n_train:]\n",
    "\n",
    "print(\"Starting ANN. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "gc.collect()\n",
    "# Cross validation model\n",
    "folds = KFold(n_splits=num_folds, shuffle=True, random_state=1001)\n",
    "# Create arrays and dataframes to store results\n",
    "oof_preds = np.zeros(train_df.shape[0])\n",
    "sub_preds = np.zeros(test_df.shape[0])\n",
    "feature_importance_df = pd.DataFrame()\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "\n",
    "#feats = [col for col in feats_0 if df[col].dtype == 'object']\n",
    "\n",
    "\n",
    "print(train_df[feats].shape)\n",
    "for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train['TARGET'])):\n",
    "        \n",
    "        categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "        \n",
    "        if encoding == 'ohe':\n",
    "            \n",
    "            enc = ce.OneHotEncoder(impute_missing=True, cols=categorical_columns).fit(train_df[feats].iloc[train_idx],\n",
    "                                                                                       train['TARGET'].iloc[train_idx])\n",
    "            x_train = enc.transform(train_df[feats].iloc[train_idx]).replace([-np.inf, np.inf], np.nan).fillna(-999)\n",
    "            x_valid = enc.transform(train_df[feats].iloc[valid_idx]).replace([-np.inf, np.inf], np.nan).fillna(-999)\n",
    "            x_test = enc.transform(test_df[feats]).replace([-np.inf, np.inf], np.nan).fillna(-999)\n",
    "            gc.collect()\n",
    "            scaler = preprocessing.RobustScaler(quantile_range=(5.0, 95.0), with_scaling=True, with_centering=True)\n",
    "            scaler.fit(x_train)\n",
    "            x_train = scaler.transform(x_train)\n",
    "            x_valid = scaler.transform(x_valid)\n",
    "            x_test = scaler.transform(x_test)\n",
    "            \n",
    "            print(x_train.shape, x_valid.shape, x_test.shape)\n",
    "        \n",
    "        file_path = \"best_model.hdf5\"\n",
    "        check_point = ModelCheckpoint(file_path, monitor=\"val_loss\", verbose=1,\n",
    "                              save_best_only=True, mode=\"min\")\n",
    "        ra_val = RocAucEvaluation(validation_data=(x_valid, train['TARGET'].iloc[valid_idx].values), interval=1)\n",
    "        early_stop = EarlyStopping(monitor=\"val_loss\", mode = \"min\", patience=1)\n",
    "        gc.collect()\n",
    "        \n",
    "        config = tf.ConfigProto(\n",
    "        intra_op_parallelism_threads=6, use_per_session_threads=6, inter_op_parallelism_threads=6)\n",
    "        with tf.Session(graph=tf.Graph(), config=config) as sess, timer('fit_predict'):\n",
    "                    ks.backend.set_session(sess)\n",
    "                    model_in = ks.Input(shape=(x_train.shape[1],), dtype='float32', sparse=False)\n",
    "                    out = ks.layers.Dense(2 ** 11,  activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(model_in)\n",
    "                    out = ks.layers.Dropout(0.5)(out)\n",
    "                    out =  ks.layers.Dense(2 ** 9, activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    out = ks.layers.Dropout(0.3)(out)\n",
    "                    out =  ks.layers.Dense(2 ** 8, activation='relu', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    out = ks.layers.Dropout(0.3)(out)\n",
    "                    out = ks.layers.Dense(1, activation='sigmoid', kernel_initializer=\n",
    "                      ks.initializers.RandomNormal(mean=0.00, stddev=0.05, seed=666))(out)\n",
    "                    model = ks.models.Model(model_in, out)\n",
    "                    model.compile(loss='binary_crossentropy',\n",
    "                                  optimizer=ks.optimizers.Adam(lr=1e-3), metrics=['binary_crossentropy'])\n",
    "                    batch_size = 2 ** 10\n",
    "                    for i in range(3):\n",
    "                        with timer('pass ' +  str(i + 1)):\n",
    "                            model.fit(x=x_train, y=train['TARGET'].iloc[train_idx].values, batch_size=batch_size+(batch_size*(2*i)),\n",
    "                                      epochs=20, validation_data=(x_valid, train['TARGET'].iloc[valid_idx].values), callbacks=[ra_val, \n",
    "                                      check_point, early_stop],\n",
    "                                      shuffle=True)\n",
    "                    y_pred = model.predict(x_valid).reshape(-1, 1)\n",
    "                    print(roc_auc_score(y_true=train['TARGET'].iloc[valid_idx].values, y_score=y_pred))\n",
    "                    gc.collect()   \n",
    "                    oof_preds[valid_idx] = model.predict(x_valid)[:, 0]\n",
    "                    sub_preds += model.predict(x_test)[:, 0] / folds.n_splits\n",
    "                    gc.collect()\n",
    "\n",
    "print('Full AUC score %.6f' % roc_auc_score(train['TARGET'], oof_preds))\n",
    "             \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "roc_auc_score(train['TARGET'], oof_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Full AUC score %.6f' % roc_auc_score(train['TARGET'], oof_preds))\n",
    "        \n",
    "sub_df = test[['SK_ID_CURR']].copy()\n",
    "sub_df['TARGET'] = sub_preds\n",
    "sub_df[['SK_ID_CURR', 'TARGET']].to_csv(test_file_path, index= False)\n",
    "\n",
    "val_df = train[['SK_ID_CURR', 'TARGET']].copy()\n",
    "val_df['TARGET'] = oof_preds\n",
    "val_df[['SK_ID_CURR', 'TARGET']].to_csv(validation_file_path, index= False)        \n",
    "            \n",
    "            \n",
    "        \n",
    "        \n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
